package com.liyuan.help_e.component.crawler;

import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.*;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.List;

@Component
public class GdufsSpider implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setDomain("www.gdufs.edu.cn");
    private static String publishTime = null;
    private String presentTime = new SimpleDateFormat("YYYY-MM-dd").format(System.currentTimeMillis());//获得2020-02-02格式

    @Override
    public void process(Page page) {
        if (page.getUrl().toString().equals("https://www.gdufs.edu.cn/gwxw.htm") || page.getUrl().regex("https://www\\.gdufs.edu\\.cn/gwxw/(.*?)\\.htm").match()) {
            if (publishTime != null && !presentTime.equals(publishTime)) {
                return;
            }
            page.addTargetRequests(page.getHtml().css("li[id^='lineu']").css("span a", "href").all());
            page.addTargetRequest(page.getHtml().css("#wrap > div.main > div.mdecondecon > div > div:nth-child(3) > table > tbody > tr > td > table > tbody > tr > td:nth-child(2) > div > a:nth-child(3)", "href").get());
        } else if (page.getUrl().regex("https://www.gdufs.edu.cn/info/1106/(.*?)\\.htm").match()) {
            publishTime = page.getHtml().css("#wrap > div.main > div.mdecondecon > form > table > tbody > tr:nth-child(2) > td > span.timestyle53824", "text").get().trim();
//            if (!publishTime.equals(presentTime)) {
//                page.setSkip(true);
//            }
            page.putField("publishTime", publishTime);
            page.putField("title", page.getHtml().css("#wrap > div.main > div.mdecondecon > form > table > tbody > tr:nth-child(1) > td", "text").get());
            page.putField("url", page.getUrl().toString());
            page.putField("author", page.getHtml().css("#wrap > div.main > div.mdecondecon > form > table > tbody > tr:nth-child(2) > td > span.authorstyle53824", "text").get());
            List<String> imageUrls = page.getHtml().$("#vsb_newscontent div.v_news_content img", "src").all();
            if(imageUrls.size()==0){
                page.putField("imageUrl","");
            }else {
                String path = "\\images\\img";
                page.putField("imageUrl",path+imageUrls.get(0));
            }
//            #n53622
            String s = page.getHtml().$(".clickstyle53824 span").toString();
//            page.putField("readCount",page.getHtml().$(".clickstyle53824 span","text"));
            new Thread(() -> {
                try {
                    downloadImages(imageUrls);
                } catch (IOException e) {
                    System.out.println("多线程下载图片!");
                    e.printStackTrace();
                }
            }).start();
//            因为图片的路径改变了，所以里面的路径也要改变！
            String content = page.getHtml().$("#vsb_newscontent div.v_news_content").get();
            page.putField("content", content);
        }
    }

    private void downloadImages(List<String> imageUrls) throws IOException {
        for (String imageUrl : imageUrls) {
            String[] fileName_ = imageUrl.split("\\?");
            String[] fileName = fileName_[0].split("/");
            String filename = fileName[fileName.length - 1];
            String dirs = fileName_[0].replace("/" + filename, "");
            File dir = new File(System.getProperty("user.dir") + "\\target\\classes\\static\\images\\img" + dirs);
            if (!dir.exists()) {
                dir.mkdirs();
            }
            imageUrl = "https://" + site.getDomain() + imageUrl;
            URL url = new URL(imageUrl);
            HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
            try {
                httpURLConnection.connect();
            } catch (ConnectException e) {
                System.out.println("这张图片连接超时了！" + imageUrl);
                e.printStackTrace();
            }
            DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(httpURLConnection.getInputStream()));
            String file = dir + "\\" + filename;
            DataOutputStream dataOutputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(file))));
            int v;
            while ((v = dataInputStream.read()) != -1) {
                dataOutputStream.write(v);
            }
            dataInputStream.close();
            dataOutputStream.close();
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

}
